package cn.wlh.learn.webmagic.processor;

import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.db.Entity;
import cn.hutool.db.Session;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import cn.wlh.learn.webmagic.pipeline.DBPipeline;
import cn.wlh.learn.webmagic.utils.ArrayUtils;
import cn.wlh.learn.webmagic.utils.SelectableUtils;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.sql.SQLException;
import java.util.List;
import java.util.Set;

/**
 * 这里少采集了，title.
 * 用hutool没法获得html内容.
 * 需要重新再采集一次.
 *
 *
 * webMagic获取cookie登陆
 * https://blog.csdn.net/weixin_42244754/article/details/81540678
 * ，账户过期了，没法用了。爬出来了，但是没法用了。
 * Selenium
 */
public class PiaochongTitleSelenium implements PageProcessor {
    private final static Log logger = LogFactory.get();

    private Site site = Site
            .me()
            .setRetryTimes(3)
            .setSleepTime(7000)
            .setRetrySleepTime(7000)
            .setTimeOut(7000)
            .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
            .addHeader("Connection", "keep-alive")
            .addHeader("X-Requested-With", "XMLHttpRequest")
            .addHeader("Content-Type",
                    "application/x-www-form-urlencoded;charset=utf-8")
            .addHeader(
                    "User-Agent",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
    // 用来存储cookie信息
    private Set<Cookie> cookies;

    @Override
    public void process(Page page) {

        // 抓取规则
        Selectable url = page.getUrl();
//        logger.warn("url=[{}]", url);


        //列表页
        Html html = page.getHtml();
            // //*[@id="aimg_7996"]
            logger.debug("详情页面内容=[{}]", html);
            // 获得图片.没有就不记录.
            List<String> images = html.xpath("//ignore_js_op").links().all();
//            boolean empty = CollectionUtil.isEmpty(images);

            // 联系电话。
            List<String> phone = SelectableUtils.xpath(html,"//div[@class='showhide']/div[@class='ck8-tips-yd']/strong","//div[@class='showhide']").all();

            // 价格
            String rmb = html.xpath("//td[@class='t_f']/strong[2]").get();
            // 服务
            String fuwu = html.xpath("//td[@class='t_f']/strong[1]").get();

            // 判断是否标准.
            boolean isBiaoZhun = ArrayUtils.orExe((list) -> {
                return CollectionUtil.isEmpty(list);
            }, images, phone);
            String body = null;
            // 不是标准获得所有内容.
            if(isBiaoZhun){
            }
            body = html.xpath("//td[@class='t_f']").get();


            // body都没有，就是垃圾数据。放弃他.
            if( StrUtil.isBlank(body)) return;
            // 放入内容存储.

            page.putField("lianxifs",CollectionUtil.join(phone,"="));
            page.putField("rmb", rmb);
            page.putField("fuwu",fuwu);
            JSONArray field = JSONUtil.parseArray(images);
            String strImages = field.toString();
            page.putField("images", strImages);
            page.putField("body",body);

            // 更新title.
            Selectable xpath = html.xpath("/html/head/meta[@name='keywords']");
            Selectable content = xpath.$("meta","content");
            String title = content.get();
            // 更新etl表
            page.putField("title",title);

//            page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2"));
//            page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));
//            page.putField("date",
//                    page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)"));
    }

    @Override
    public Site getSite() {

        // 将获取到的cookie信息添加到webmagic中
        for (Cookie cookie : cookies) {
            site.addCookie(cookie.getName().toString(), cookie.getValue()
                    .toString());
        }

        return site;
    }


    public final static String URL = "https://www.snnna.com/forum.php?mod=forumdisplay&fid=53&page=1"; // 地址

    public static final String URL_LIST = "https://www.snnna.com/forum.php?mod=forumdisplay&fid=53&page=";

    // https://www.snnna.com/forum.php?mod=viewthread&tid=3901&extra=page%3D1
//    public static final String URL_POST = "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";

    //  https://www.snnna.com/forum.php?mod=redirect&tid=2055&goto=lastpost#lastpost
//    public static final String URL_POST = "https://www.snnna.com/forum.php\\?mod=viewthread&tid=\\w+\\&extra=page%3D1";
    public static final String URL_POST = "https://www.snnna.com/forum.php\\?.*";

    public void login() {
        // 登陆
        System.setProperty("webdriver.chrome.driver",
                "D:/chromedriver.exe"); // 注册驱动
        WebDriver driver = new ChromeDriver();


        driver.get(URL);// 打开网址

        // 防止页面未能及时加载出来而设置一段时间延迟
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        // 设置用户名密码
        driver.findElement(By.id("ls_username")).sendKeys("873406903"); // 用户名
        driver.findElement(By.id("ls_password")).sendKeys("873406903"); // 密码
        // 模拟点击
        driver.findElement(By.xpath("//*[@id=\"lsform\"]/div/div/table/tbody/tr[2]/td[3]/button"))
                .click(); // xpath语言：id为form-group-login的form下的button

        // 防止页面未能及时加载出来而设置一段时间延迟
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        // 获取cookie信息
        cookies = driver.manage().getCookies();
        System.out.println("cookies====================="+cookies);
        driver.close();
    }
    public static int PAGE = 11;
    public static void main(String[] args) throws SQLException {

        PiaochongTitleSelenium dome = new PiaochongTitleSelenium();
        dome.login(); // 登陆
        // 插入到。sheji_caiji_piaochong 表中.
        DBPipeline dBPipeline = new DBPipeline("sheji_caiji_piaochong");
        dBPipeline.setInsertOrUpdate(true);
        Spider spider = Spider.create(dome);

        Session session = dBPipeline.getSession();

        Entity entity = new Entity("sheji_etl_piaochong");
        entity.set("title",null);
        int numPerPage = 20;
        long count = session.count(entity);
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                String url = piaochong.getStr("id");
                spider.addUrl(url);
            }
        }
        // 爬虫
        spider.addPipeline(dBPipeline).thread(1).start();
    }
}